import os
import logging
import urllib.request

logdir = "./logs"
if not os.path.isdir(logdir):
    os.mkdir(logdir)
logging.basicConfig(
    # filename="./logs/access.log",
    level=logging.DEBUG,
    format="%(asctime)s %(name)s %(levelname)s %(message)s",
    datefmt='%Y-%m-%d  %H:%M:%S %a'
)
logging.info("start application.....")

filterNames = {'论文名称', '../', '论文地址'}
fileExt={".ppt",".pptx",".pdf",".mov",".jpg",'.ps'}

# 文件名称是否是被过滤的
# 是: 这个文件不要
# 否: 要
def isInSet(fileName):
    for filterName in filterNames:
        if filterName == fileName:
            return True
    return False


def processFileName(param):
    fileName=  param.replace("..","").replace(">","")
    for name in fileExt:
        if fileName.endswith(name):
            return fileName
        # else:
        #     return fileName+".pdf"
    return fileName+".pdf"


def downloadFile(items):
    # 文件地址
    fileServerAddr = items[2]

    # dir
    dirName=items[0]
    # 文件名
    fileName = items[1]
    # 如果不加上下面的这行出现会出现urllib2.HTTPError: HTTP Error 403: Forbidden 错误
    # 主要是由于该网站禁止爬虫导致的，可以在请求加上头信息，伪装成浏览器访问User-Agent,具体的信息可以通过火狐的FireBug插件查询

    dirNames = dirName.split("/")
    targetFileSaveDir = "D:\\AiOps\\" + dirNames[3] + "\\"
    os.makedirs(targetFileSaveDir, mode=777, exist_ok=True)
    targetFileName = processFileName(targetFileSaveDir + fileName)
    if os.path.exists(targetFileName):
        logging.info("file is existed ,passed for "+targetFileName)
        return
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.101 Safari/537.36 Edg/91.0.864.48'}
    try:

        logging.debug("start  downloading for "+fileServerAddr +" ...")
        req = urllib.request.Request(url=fileServerAddr, headers=headers)
        f = urllib.request.urlopen(req).read()


        logging.info("save file to " + targetFileName)
        with open(targetFileName, "wb") as code:
            code.write(f)
        logging.debug("end  downloading...")
        logging.debug("=======================================================================")
    except Exception:
        logging.error(fileServerAddr +"is error, passed it")



def processCsv(items):
    # 文件地址
    fileServerAddr = items[2]

    # 文件名
    fileName = items[1]
    # print(fileName)
    #     论文名称
    # ../
    # 2016CCR_KnowledgeDefinedNetworking.pdf
    # AIOpsWorkshopKeynote1.3publicw.pdf
    # CourseIntro_Spring2021.pdf
    # Internet_basics.ppt

    if isInSet(fileName):
        logging.debug("pass for file " + fileName)
    else:
        #         download
        downloadFile(items)


def readCsv(file):
    with open(file) as f:
        line = f.readline();
        while (line):
            items = line.split(",")
            line = f.readline()
            processCsv(items)
